# -*- coding: utf-8 -*-
"""
Created on Fri Dec 04 19:57:57 2015

@author: liangrongli
"""
# 在已经采集的URL列表中抓取python.jobbole.com的教程内容


def remove_tag(str):
    import re
    
    #删除 导航菜单
    rm_nav = re.compile('<nav .*?>.*?</nav>',re.S)
    str = re.sub(rm_nav,'',str)
    
    #删除 iframe
    rm_iframe = re.compile('<iframe .*?>.*?</iframe>',re.S)
    str = re.sub(rm_iframe,'',str)
    
    #删除 header
    rm_header = re.compile('<header>.*?</header>',re.S)
    str = re.sub(rm_header,'',str)
    
    #删除 breadcrumb
    rm_breadcrumb = re.compile('<div id="breadcrumb" .*?>.*?</div></div>',re.S)
    str = re.sub(rm_breadcrumb,'',str)
    
    #删除 sidebar - <footer>
    rm_footer = re.compile('<div id="sidebar" .*?>(.*?)</div>.*?<footer>(.*?)</footer>',re.S)
    str = re.sub(rm_footer,'</div>',str)
    
    #删除 adds-interest
    rm_adds = re.compile('<div class="post-adds">.*?<div style="text-align: left;">.*?</div>',re.S)
    str = re.sub(rm_adds,'</div></div>',str)
    
    #删除 jiathis
    rm_jiathis = re.compile('<div class="jiathis_style" .*?>.*?</div>',re.S)
    str = re.sub(rm_jiathis,'',str)
    
    #删除 comment to end
    rm_comment = re.compile('<div id="article-comment" .*?>.*?<div id="comment_list" .*?>.*?</div>',re.S)
    str = re.sub(rm_comment,'',str)
    
    #替换为全屏显示
    str = re.sub('grid-8','grid-12',str)
    
    return str.strip()
    
#获取原网页源代码
def get_source(url):
    import urllib2
    
    #url = "http://python.jobbole.com/43922/"
    user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36"
    headers = {"User-Agent":user_agent}

    req = urllib2.Request(url, None, headers)
    resp = urllib2.urlopen(req, timeout=20)
    return resp.read()
    
#获取原网页title
def get_title(url):
    import re
    pattern = re.compile('<h1>(.*?)</h1>')
    page_source = get_source(url)
    title = re.search(pattern,page_source)
    if title:
        return title.group(1)
    else:
        return None

#获取原网页过滤后的html内容
def get_html(url):
    page_source = get_source(url)
    filter_html = remove_tag(page_source)
    if filter_html:
        return filter_html
    else:
        return None
    
if __name__ == '__main__':
    import time
    import re
    p = re.compile('([\d]{5})')
    start = time.clock()
    
    
    with open('url_list.txt','r') as url_list:
        for url in url_list:            
            #title = get_title(url)
            title = re.search(p,url)
            print title.group(1)
            file_name = 'python_material/'+title.group(1)+'.html'
            print file_name
            html = get_html(url)
            f = open(file_name,'w')
            f.write(html)
            f.closed
    url_list.closed
    
    end = time.clock()
    print "run time is %f s" % (end-start)
    
    #print dir(top_nav)
    #直接把网页保存为本地文件
    #urllib.urlretrieve(url,'43922.html')

